package navigation.results.cluster.follow;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileFilter;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;

import navigation.textTransformer.NumberReplaceMent;
import navigation.textTransformer.TextTransformer;

import org.htmlparser.Node;
import org.htmlparser.Parser;
import org.htmlparser.filters.HasAttributeFilter;
import org.htmlparser.filters.OrFilter;
import org.htmlparser.lexer.Lexer;
import org.htmlparser.lexer.Page;
import org.htmlparser.nodes.TagNode;
import org.htmlparser.util.NodeList;
import org.htmlparser.util.ParserException;
import org.lobobrowser.html.domimpl.HTMLElementBuilder.Td;

import wish.stages.ParsingStage;

public class FollowAnalyzer {
	
	private static final String HREF_ATTRIBUTE = "href";
	private static final String ONCLICK_ATTRIBUTE = "onclick";
	private static final String COMB_ATTRIBUTE = "combined";
	
	public class FollowAnalyzerRunnable implements Runnable{

		
		private File[] follows;
		private File output;
		private BufferedWriter report;
		private List<String> queries;
		private TextTransformer tt;

		public FollowAnalyzerRunnable(File[] follows, File output, BufferedWriter bw, List<String> tfidfs, TextTransformer tt) {
			this.follows = follows;
			this.output = output;
			this.report = bw;
			this.queries = tfidfs;
			this.tt = tt;
		}

		@Override
		public void run() {
			
			Map<String,List<String>> hypMap = new HashMap<String, List<String>>();
			
			Map<String, List<String>> textMap = new HashMap<String,List<String>>();
			
			Map<String, NodeList> nodesMap = new HashMap<String, NodeList>();
			
			//Load The Structures
			
			for (int i = 0; i < follows.length; i++) {
							
				try {
					
					int index = Integer.valueOf(follows[i].getName().replaceAll(".html", ""));
					
					String qur = "";
					
					if (index>=0)
						qur = queries.get(index).toLowerCase();
					
					String htmlContent = ParsingStage.readFile(follows[i]);
					
					if (htmlContent==null){
						System.out.println(follows[i]);
					}
					
					Parser parser = new Parser(new Lexer(new Page(htmlContent, "UTF-8")));
					
					NodeList n = parser.parse(new OrFilter(new HasAttributeFilter(HREF_ATTRIBUTE),new HasAttributeFilter(ONCLICK_ATTRIBUTE)));
					
					for (int j = 0; j < n.size(); j++) {
						
						TagNode node = (TagNode)n.elementAt(j);
						
						String text = ParsingStage.recoverText(node,true,tt.personalize(qur));
						
						updateMap(text,textMap,follows[i].getName());
						
						String hyperLink = node.getAttribute(HREF_ATTRIBUTE);

						String onclick = node.getAttribute(ONCLICK_ATTRIBUTE);
						
						String combined = combine(hyperLink,onclick);
						
						updateMap(combined,hypMap,follows[i].getName());
													
					}
					
					nodesMap.put(follows[i].getName(), n);
					
				} catch (IOException e) {
					// TODO Auto-generated catch block
					e.printStackTrace();
				} catch (ParserException e) {
					// TODO Auto-generated catch block
					e.printStackTrace();
				}
				
			}			
			
			Map<String,Map<String,Integer>> textFreq = generateTextFreqStructure(textMap);
			
			Map<String,Map<String,Integer>> hypFreq = generateTextFreqStructure(hypMap);
			
			try {
				
				BufferedWriter bw = new BufferedWriter(new FileWriter(output));
				
				int numberOfTheCluster = 1;
				
				int noCluster = 1;
								
				for (int i = 0; i < follows.length; i++) {
					
					int index = Integer.valueOf(follows[i].getName().replaceAll(".html", ""));
					
					String qur = "";
					
					if (index>=0)
						qur = queries.get(index).toLowerCase();
					
					NodeList nl = nodesMap.get(follows[i].getName());
					
					bw.write("<br><h2> File: " + follows[i].getName() + "</h2>");
					
					boolean theClusterFound = false;
					
					for (int j = 0; j < nl.size(); j++) {
						
						TagNode node = (TagNode)nl.elementAt(j);
						
						String text = ParsingStage.recoverText(node,true,tt.personalize(qur));
						
						String href = node.getAttribute(HREF_ATTRIBUTE);

						String onclick = node.getAttribute(ONCLICK_ATTRIBUTE);
						
						if (onclick == null)
							if (!href.startsWith("javascript:") && !generateWords(href.toLowerCase()).contains(qur))
								continue;
								
						String combined = combine(href, onclick);
									
						boolean javascript = false;
						
						if (href.startsWith("javascript:") || onclick != null)
							javascript = true;
						
						if (javascript){
							
							boolean samefreqInAll = sameFrequencyInAll(textFreq.get(text)); 
							
							boolean isbelow = isAlwaysBelowFrequency(hypFreq.get(combined),4);
							
							boolean istextbelow = isAlwaysBelowFrequency(textFreq.get(text), 4);
							
							boolean next = text.toLowerCase().contains("next");
							
							boolean page = generateWords(combined.toLowerCase()).contains("page");
							
							if (isbelow && istextbelow && samefreqInAll && (next || page)){
								
								report.write("<br>" + node.toHtml());
								
							}
							
						} else {
							
							int freqHyp = getFrequency(hypMap.get(combined));
							
							bw.write("<div");
							
							if (freqHyp == 1){
								
								int freqText = getFrequency(textMap.get(text));
								
								if (freqText > 1){
								
									boolean samefreqTextInDoc = isAlwaysBelowFrequency(textFreq.get(text),4);
									
									boolean inAll = isInAll(textFreq.get(text), follows.length);
									
									if (samefreqTextInDoc && !inAll){
									
										if (!theClusterFound)
											report.write("<br><h2> Query: " + qur+"</h2>");
										
										report.write("<br>" + node.toHtml());
										
										bw.write(" id=\""+ numberOfTheCluster + "\"><h1> THE CLUSTER: </h1>\n");
										bw.write("<h2> Cluster: " + numberOfTheCluster + "</h2>");
										bw.write("<h2> Text: " + text + "</h2>");
										bw.write("<h2>" + textMap.get(text).size() +"</h2>");
										bw.write("<h2>" + hypMap.get(combined).size() + "</h2>");
										theClusterFound = true;
										numberOfTheCluster++;
	
									}else{
										bw.write(" id=\"" + noCluster + "\">\n<h2> Cluster: " + noCluster + "</h2>\n");
										noCluster++;
									}
								}else{
									bw.write(" id=\"" + noCluster + "\">\n<h2> Cluster: " + noCluster + "</h2>\n");
									noCluster++;
								}
							}else{
								bw.write(" id=\"" + noCluster + "\">\n<h2> Cluster: " + noCluster + "</h2>\n");
								noCluster++;
							}
												
							bw.write("<br>");
							
							bw.write(node.toHtml());
								
							bw.write("</div>");
								
							}
					
						}
					
				}
				
//				if (!theClusterFound){
//					report.write("\n<br><h2> Not Found" + output.getName() + "</h2>");
//				}
				
				bw.close();
				
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			} 
		}

		private boolean sameFrequencyInAll(Map<String, Integer> map) {
			
			int prev = -1;			
			
			for (Entry<String,Integer> entry : map.entrySet()) {
				
				if (prev==-1){
					prev = entry.getValue();
				}else{
					if (prev != entry.getValue())
						return false;
					else
						prev = entry.getValue();
				}
				
			}
			
			return true;
		}

		private List<String> generateWords(String text) {
			
			String[] spl = text.split("\\W");
			
			return Arrays.asList(spl);
			
		}

		private boolean isInAll(Map<String, Integer> map, int length) {
			
//			return (double)map.keySet().size() > ((double)length - (0.06)*(double)length);
			
			return map.keySet().size() == length;
			
		}

		private boolean isAlwaysBelowFrequency(Map<String, Integer> map, int freq) {
			
			for (Entry<String,Integer> entry : map.entrySet()) {
				
				if (entry.getValue() > freq)
					return false;
				
			}
			
			return true;
			
		}

		private Map<String, Map<String, Integer>> generateTextFreqStructure(
				Map<String, List<String>> textMap) {

			Map<String, Map<String,Integer>> ret = new HashMap<String, Map<String,Integer>>();
				
			for (Entry<String,List<String>> entry : textMap.entrySet()) {
				
				Map<String,Integer> map = generateFreqMap(entry.getValue());
				
				ret.put(entry.getKey(), map);
				
			}
			
			return ret;
		}

		private Map<String, Integer> generateFreqMap(List<String> value) {
			
			Map<String,Integer> freqMap = new HashMap<String, Integer>();
			
			for (String string : value) {
				
				Integer freq = freqMap.get(string);
				
				if (freq == null){
					freq = 0;
				}
				
				freqMap.put(string, freq+1);
			}
			
			return freqMap;
		}

		

		private String combine(String hyperLink, String onclick) {
			
			if (onclick == null)
				return hyperLink;
			if (hyperLink == null)
				return onclick;
			
			return hyperLink + " - " + onclick;
		}

		private int getFrequency(List<String> list) {
			
			Set<String> set = new HashSet<String>();
			
			for (String string : list) {
				set.add(string.trim());
			}

			return set.size();
			
		}

		private void updateMap(String text, Map<String, List<String>> textMap,
				String name) {
			
			List<String> files = textMap.get(text);
			
			if (files == null){
				files = new ArrayList<String>();
				textMap.put(text, files);
			}
			
			files.add(name);
			
		}
		
	}

	private static Set<String> toProcess;
	
	public static void main(String[] args) throws IOException {
		
		toProcess = loadToProcess("/proj/dbNoBackup/pjbarrio/Experiments/Wrappers/training");
		
		String type = "training";
		
		File nav = new File("/proj/dbNoBackup/pjbarrio/Experiments/Wrappers/Navigation/" + type + "/");
		
		File navEmpty = new File("/proj/dbNoBackup/pjbarrio/Experiments/Wrappers/NavigationEmpty/" + type + "/");
		
		File outFold = new File("/proj/dbNoBackup/pjbarrio/Experiments/Wrappers/NavigationNext/" + COMB_ATTRIBUTE + "/" + type + "/");
		
		File outReport = new File("/proj/dbNoBackup/pjbarrio/Experiments/Wrappers/NavigationNext/" + COMB_ATTRIBUTE + "." + type + "report-");
		
		String tfidffolder = "/proj/dbNoBackup/pjbarrio/Experiments/Wrappers/tfidf/"+type+"/";
		
		FileFilter ff = new FileFilter() {
			
			@Override
			public boolean accept(File pathname) {
				
				return toProcess.contains(pathname.getName());
				
			}
		};
		
		File[] emptyFiles = navEmpty.listFiles(ff);
		
		File[] files = nav.listFiles(ff);
		
		TextTransformer tt = new NumberReplaceMent();
		
//		files = new File[1];
//		
//		files[0] = new File("/proj/dbNoBackup/pjbarrio/Experiments/Wrappers/Navigation/" + type + "/17/");
		
		Arrays.sort(files,new Comparator<File>() {

			@Override
			public int compare(File o1, File o2) {
				
				return Double.compare(Double.valueOf(o1.getName()), Double.valueOf(o2.getName()));
				
			}
		});
		
		BufferedWriter bw = new BufferedWriter(new FileWriter(outReport));

		int hh= 0;
		
		for (int i = 0; i < files.length; i++) {
			
			if (i % 40 == 0){
				
				bw.close();
				bw = new BufferedWriter(new FileWriter(outReport.getAbsolutePath() + hh + ".html"));
				hh++;
				
			}
						
			System.out.println("DATABASE: " + i + " - " + files[i].getName());
			
			bw.write("<h2> Database: " + files[i].getName() + "</h2>");
			
			System.gc();
			
			File folder = files[i];
			
			File[] follows = combine(folder.listFiles(),getFiles(emptyFiles,files[i].getName()));
			
			File output = new File(outFold,folder.getName() + ".html");
			
			List<String> tfidfs = loadQueries(new File(tfidffolder,files[i].getName()));
			
			new FollowAnalyzer().execute(follows,output,bw,tfidfs,tt);
			
		}
		
		bw.close();
	}

	private static File[] getFiles(File[] emptyFiles, String name) {
		
		for (int i = 0; i < emptyFiles.length; i++) {
			
			if (name.equals(emptyFiles[i].getName())){
				return emptyFiles[i].listFiles();
			}
			
		}
		
		return null;
	}

	private static File[] combine(File[] files1, File[] files2) {
		
		File[] ret = new File[files1.length + files2.length];
		
		for (int i=0; i < files1.length; i++) {
			ret[i] = files1[i];
		}
		
		for (int i = 0; i < files2.length; i++) {
			ret[files1.length + i] = files2[i];
		}
		
		return ret;
		
	}


	private static Set<String> loadToProcess(String fileName) throws IOException {
		
		Set<String> set = new HashSet<String>();
		
		BufferedReader br = new BufferedReader(new FileReader(fileName));
		
		String line;
		
		while((line=br.readLine())!=null){
			
			set.add(line);
			
		}
		
		br.close();
		
		return set;
		
	}
	
	private static List<String> loadQueries(File file) throws IOException {
		
		List<String> ret = new ArrayList<String>();
		
			
		BufferedReader br = new BufferedReader(new FileReader(file));
		
		String line;
		
		while ((line=br.readLine())!=null){
			
			String[] l = line.split(" ");
			
			ret.add(l[0]);
			
		}
		
		br.close();
		
		return ret;
		
	}
	
	private void execute(File[] follows, File output, BufferedWriter bw, List<String> tfidfs, TextTransformer tt) {
		
		Thread t = new Thread(new FollowAnalyzerRunnable(follows,output,bw, tfidfs,tt));
		
		t.start();
		
		try {
			t.join();
		} catch (InterruptedException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		
	}
	
}
